{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using ReinforcementLearningAnIntroduction\n",
    "\n",
    "using StatsBase\n",
    "\n",
    "using Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{Float64,1}:\n",
       " 0.16666666666666666\n",
       " 0.3333333333333333 \n",
       " 0.5                \n",
       " 0.6666666666666666 \n",
       " 0.8333333333333334 "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "true_values = [i/6 for i in 1:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7, 2)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = RandomWalkEnv(leftreward=0.)\n",
    "n_state, n_action = length(get_observation_space(env)), length(get_action_space(env))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "struct RecordRMS <: AbstractHook\n",
    "    rms::Vector{Float64}\n",
    "    RecordRMS() = new([])\n",
    "end\n",
    "\n",
    "(f::RecordRMS)(::PostEpisodeStage, agent, env, obs) = push!(f.rms, sqrt(mean((agent.policy.learner.approximator.table[2:end - 1] - true_values).^2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "create_MC_agent (generic function with 1 method)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "create_TD_agent(α) = Agent(\n",
    "    policy=VBasedPolicy(\n",
    "        learner = TDLearner(\n",
    "            approximator=TabularApproximator(fill(0.5, n_state)),\n",
    "            optimizer=Descent(α),\n",
    "            method=:SRS\n",
    "        ),\n",
    "        mapping = (obs, V) -> begin\n",
    "            value = zeros(n_action)\n",
    "            value[rand(1:n_action)] = 1.0\n",
    "            value\n",
    "        end\n",
    "    ),\n",
    "    trajectory=EpisodicCompactSARTSATrajectory()\n",
    ")\n",
    "\n",
    "create_MC_agent(α) = Agent(\n",
    "    policy=VBasedPolicy(\n",
    "        learner=MonteCarloLearner(\n",
    "            approximator=TabularApproximator(fill(0.5, n_state)),\n",
    "            α=α,\n",
    "            kind=EVERY_VISIT\n",
    "            ),\n",
    "        mapping = (obs, V) -> begin\n",
    "            value = zeros(n_action)\n",
    "            value[rand(1:n_action)] = 1.0\n",
    "            value\n",
    "        end\n",
    "        ),\n",
    "    trajectory=EpisodicCompactSARTSATrajectory()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip2500\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip2500)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip2501\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip2500)\" d=\"\n",
       "M153.898 1487.47 L2352.76 1487.47 L2352.76 47.2441 L153.898 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip2502\">\n",
       "    <rect x=\"153\" y=\"47\" width=\"2200\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  216.13,1487.47 216.13,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  734.728,1487.47 734.728,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1253.33,1487.47 1253.33,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1771.93,1487.47 1771.93,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2290.52,1487.47 2290.52,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1353.08 2352.76,1353.08 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,1161.77 2352.76,1161.77 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,970.449 2352.76,970.449 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,779.132 2352.76,779.132 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,587.814 2352.76,587.814 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,396.497 2352.76,396.497 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  153.898,205.179 2352.76,205.179 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1487.47 153.898,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,1487.47 216.13,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  734.728,1487.47 734.728,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1253.33,1487.47 1253.33,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1771.93,1487.47 1771.93,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2290.52,1487.47 2290.52,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1353.08 180.284,1353.08 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,1161.77 180.284,1161.77 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,970.449 180.284,970.449 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,779.132 180.284,779.132 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,587.814 180.284,587.814 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,396.497 180.284,396.497 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  153.898,205.179 180.284,205.179 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 216.13, 1541.47)\" x=\"216.13\" y=\"1541.47\">1</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 734.728, 1541.47)\" x=\"734.728\" y=\"1541.47\">2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1253.33, 1541.47)\" x=\"1253.33\" y=\"1541.47\">3</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1771.93, 1541.47)\" x=\"1771.93\" y=\"1541.47\">4</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2290.52, 1541.47)\" x=\"2290.52\" y=\"1541.47\">5</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1370.58)\" x=\"129.898\" y=\"1370.58\">0.2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 1179.27)\" x=\"129.898\" y=\"1179.27\">0.3</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 987.949)\" x=\"129.898\" y=\"987.949\">0.4</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 796.632)\" x=\"129.898\" y=\"796.632\">0.5</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 605.314)\" x=\"129.898\" y=\"605.314\">0.6</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 413.997)\" x=\"129.898\" y=\"413.997\">0.7</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 129.898, 222.679)\" x=\"129.898\" y=\"222.679\">0.8</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,874.79 734.728,779.132 1253.33,779.132 1771.93,779.132 2290.52,779.132 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,1239.7 734.728,976.277 1253.33,857.5 1771.93,769.994 2290.52,612.962 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#3da44d; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,1446.71 734.728,1151.64 1253.33,877.47 1771.93,351.389 2290.52,88.0053 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2502)\" style=\"stroke:#c271d2; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  216.13,1416.86 734.728,1097.99 1253.33,779.132 1771.93,460.269 2290.52,141.407 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip2500)\" d=\"\n",
       "M1737.03 1343.47 L2280.76 1343.47 L2280.76 1041.07 L1737.03 1041.07  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1737.03,1343.47 2280.76,1343.47 2280.76,1041.07 1737.03,1041.07 1737.03,1343.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1761.03,1101.55 1905.03,1101.55 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1929.03, 1119.05)\" x=\"1929.03\" y=\"1119.05\">episode = 1</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1761.03,1162.03 1905.03,1162.03 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1929.03, 1179.53)\" x=\"1929.03\" y=\"1179.53\">episode = 10</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#3da44d; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1761.03,1222.51 1905.03,1222.51 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1929.03, 1240.01)\" x=\"1929.03\" y=\"1240.01\">episode = 100</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2500)\" style=\"stroke:#c271d2; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1761.03,1282.99 1905.03,1282.99 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2500)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1929.03, 1300.49)\" x=\"1929.03\" y=\"1300.49\">true value</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = plot(;legend=:bottomright)\n",
    "for i in [1, 10, 100]\n",
    "    agent = create_TD_agent(0.1)\n",
    "    run(agent, env, StopAfterEpisode(i))\n",
    "    plot!(p, agent.policy.learner.approximator.table[2:end - 1], label=\"episode = $i\")\n",
    "end\n",
    "plot!(p, true_values, label=\"true value\")\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mProgress: 100%|█████████████████████████████████████████| Time: 0:00:00\u001b[39m\n",
      "\u001b[32mProgress: 100%|█████████████████████████████████████████| Time: 0:00:00\u001b[39m\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip2900\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip2900)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip2901\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip2900)\" d=\"\n",
       "M180.66 1487.47 L2352.76 1487.47 L2352.76 47.2441 L180.66 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip2902\">\n",
       "    <rect x=\"180\" y=\"47\" width=\"2173\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  221.436,1487.47 221.436,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  738.898,1487.47 738.898,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1256.36,1487.47 1256.36,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1773.82,1487.47 1773.82,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2291.28,1487.47 2291.28,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,1327.04 2352.76,1327.04 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,988.935 2352.76,988.935 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,650.833 2352.76,650.833 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,312.731 2352.76,312.731 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,1487.47 180.66,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  221.436,1487.47 221.436,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  738.898,1487.47 738.898,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1256.36,1487.47 1256.36,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1773.82,1487.47 1773.82,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2291.28,1487.47 2291.28,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,1327.04 206.725,1327.04 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,988.935 206.725,988.935 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,650.833 206.725,650.833 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,312.731 206.725,312.731 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 221.436, 1541.47)\" x=\"221.436\" y=\"1541.47\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 738.898, 1541.47)\" x=\"738.898\" y=\"1541.47\">25</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1256.36, 1541.47)\" x=\"1256.36\" y=\"1541.47\">50</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1773.82, 1541.47)\" x=\"1773.82\" y=\"1541.47\">75</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2291.28, 1541.47)\" x=\"2291.28\" y=\"1541.47\">100</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 1344.54)\" x=\"156.66\" y=\"1344.54\">0.05</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 1006.44)\" x=\"156.66\" y=\"1006.44\">0.10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 668.333)\" x=\"156.66\" y=\"668.333\">0.15</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 330.231)\" x=\"156.66\" y=\"330.231\">0.20</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  242.135,118.017 262.833,163.162 283.532,207.221 304.23,249.23 324.929,290.248 345.627,331.675 366.325,370.544 387.024,409.593 407.722,448.207 428.421,484.749 \n",
       "  449.119,518.749 469.818,553.284 490.516,586.957 511.215,619.01 531.913,651.524 552.612,682.482 573.31,709.401 594.008,739.023 614.707,766.605 635.405,793.728 \n",
       "  656.104,819.038 676.802,843.8 697.501,868.179 718.199,889.54 738.898,911.941 759.596,932.972 780.294,951.679 800.993,971.406 821.691,992.657 842.39,1013.53 \n",
       "  863.088,1032.74 883.787,1051.16 904.485,1067.37 925.184,1085.41 945.882,1104.6 966.581,1119.9 987.279,1135.16 1007.98,1151.88 1028.68,1165.36 1049.37,1178.51 \n",
       "  1070.07,1191.11 1090.77,1206.95 1111.47,1220.37 1132.17,1229.33 1152.87,1245.71 1173.57,1260.48 1194.26,1268.35 1214.96,1277.05 1235.66,1287.05 1256.36,1294.27 \n",
       "  1277.06,1303.43 1297.76,1313.41 1318.45,1320.68 1339.15,1325.5 1359.85,1329.28 1380.55,1340.45 1401.25,1351.53 1421.95,1357.98 1442.64,1363.25 1463.34,1375.47 \n",
       "  1484.04,1380.5 1504.74,1386.19 1525.44,1385.74 1546.14,1387.97 1566.84,1385.7 1587.53,1394.83 1608.23,1402.67 1628.93,1402.31 1649.63,1404.98 1670.33,1407.73 \n",
       "  1691.03,1412.32 1711.72,1415.85 1732.42,1421.73 1753.12,1423.42 1773.82,1432.09 1794.52,1436.82 1815.22,1431.79 1835.92,1430.66 1856.61,1434.93 1877.31,1433.73 \n",
       "  1898.01,1438.47 1918.71,1439.16 1939.41,1438.87 1960.11,1441.22 1980.8,1443.99 2001.5,1441.25 2022.2,1438.94 2042.9,1438.52 2063.6,1437.74 2084.3,1441.06 \n",
       "  2105,1441.79 2125.69,1440.97 2146.39,1437.92 2167.09,1435.91 2187.79,1435.52 2208.49,1438.07 2229.19,1444.08 2249.88,1442.18 2270.58,1443.2 2291.28,1446.71 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  242.135,162.37 262.833,247.769 283.532,328.06 304.23,406.731 324.929,479.278 345.627,547.669 366.325,612.433 387.024,672.522 407.722,728.109 428.421,788.316 \n",
       "  449.119,844.086 469.818,896.604 490.516,944.616 511.215,993.782 531.913,1032.98 552.612,1064.89 573.31,1102.53 594.008,1138.33 614.707,1154.32 635.405,1181.99 \n",
       "  656.104,1204.68 676.802,1227.47 697.501,1246.33 718.199,1264.87 738.898,1286.21 759.596,1296.25 780.294,1325.99 800.993,1341.4 821.691,1355.56 842.39,1356.23 \n",
       "  863.088,1362.69 883.787,1365.89 904.485,1365.64 925.184,1363.28 945.882,1359.16 966.581,1361.46 987.279,1355.57 1007.98,1345.57 1028.68,1348.67 1049.37,1350.31 \n",
       "  1070.07,1347.09 1090.77,1344.38 1111.47,1336.76 1132.17,1335.34 1152.87,1334.51 1173.57,1331.03 1194.26,1328.86 1214.96,1320.38 1235.66,1326.72 1256.36,1327.96 \n",
       "  1277.06,1319.26 1297.76,1321.8 1318.45,1316.79 1339.15,1319.45 1359.85,1325.62 1380.55,1325.53 1401.25,1326.29 1421.95,1324.96 1442.64,1323.67 1463.34,1310.75 \n",
       "  1484.04,1311.26 1504.74,1307.75 1525.44,1306.19 1546.14,1318.08 1566.84,1319.57 1587.53,1314.6 1608.23,1302.69 1628.93,1285.77 1649.63,1299.5 1670.33,1303.82 \n",
       "  1691.03,1291.67 1711.72,1294.59 1732.42,1299.67 1753.12,1301.57 1773.82,1284.03 1794.52,1282.26 1815.22,1291.67 1835.92,1287.03 1856.61,1298.45 1877.31,1293.87 \n",
       "  1898.01,1289.44 1918.71,1291.91 1939.41,1299.92 1960.11,1298.25 1980.8,1290.23 2001.5,1298.09 2022.2,1303.68 2042.9,1304.65 2063.6,1296.44 2084.3,1288.87 \n",
       "  2105,1292.14 2125.69,1293.37 2146.39,1298.66 2167.09,1303.7 2187.79,1300.46 2208.49,1298.25 2229.19,1296.64 2249.88,1292.5 2270.58,1297.48 2291.28,1288.7 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#3da44d; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  242.135,204.154 262.833,327.131 283.532,438.898 304.23,550.046 324.929,647.882 345.627,742.34 366.325,827.572 387.024,899.301 407.722,965.625 428.421,1020.39 \n",
       "  449.119,1072.08 469.818,1120.91 490.516,1151.47 511.215,1181.78 531.913,1209.42 552.612,1227.3 573.31,1232.57 594.008,1246.21 614.707,1260.99 635.405,1266.56 \n",
       "  656.104,1287.28 676.802,1268.73 697.501,1276.12 718.199,1290.3 738.898,1301.59 759.596,1309.94 780.294,1292.78 800.993,1276.1 821.691,1285.63 842.39,1284.9 \n",
       "  863.088,1275.19 883.787,1275.25 904.485,1262.55 925.184,1263.26 945.882,1255.98 966.581,1238.29 987.279,1221.34 1007.98,1215.28 1028.68,1223.23 1049.37,1221.16 \n",
       "  1070.07,1206.8 1090.77,1199.79 1111.47,1202.49 1132.17,1206.02 1152.87,1198.84 1173.57,1199.84 1194.26,1191 1214.96,1193.25 1235.66,1191.92 1256.36,1181.03 \n",
       "  1277.06,1181.36 1297.76,1194.08 1318.45,1192.25 1339.15,1192.27 1359.85,1185.61 1380.55,1193.95 1401.25,1196.34 1421.95,1197.94 1442.64,1198.48 1463.34,1191.39 \n",
       "  1484.04,1196.04 1504.74,1198.74 1525.44,1199.08 1546.14,1198.96 1566.84,1212.34 1587.53,1207.04 1608.23,1207.72 1628.93,1212.86 1649.63,1209.15 1670.33,1196.11 \n",
       "  1691.03,1198.57 1711.72,1199.58 1732.42,1201.39 1753.12,1207.89 1773.82,1196.99 1794.52,1199.7 1815.22,1213.01 1835.92,1210.75 1856.61,1192.86 1877.31,1189.34 \n",
       "  1898.01,1190.12 1918.71,1181.03 1939.41,1187.37 1960.11,1192.28 1980.8,1190.63 2001.5,1188.88 2022.2,1189.46 2042.9,1178.87 2063.6,1181.97 2084.3,1190.22 \n",
       "  2105,1215.03 2125.69,1216.66 2146.39,1210.74 2167.09,1209.88 2187.79,1198.18 2208.49,1190.68 2229.19,1192.33 2249.88,1189.7 2270.58,1192.22 2291.28,1194.11 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#c271d2; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  242.135,88.0053 262.833,104.431 283.532,119.66 304.23,130.101 324.929,142.28 345.627,154.72 366.325,165.426 387.024,177.186 407.722,190.048 428.421,201.859 \n",
       "  449.119,214.586 469.818,225.648 490.516,237.03 511.215,250.434 531.913,262.67 552.612,274.113 573.31,285.313 594.008,299.672 614.707,312.057 635.405,325.158 \n",
       "  656.104,337.644 676.802,350.734 697.501,364.238 718.199,375.303 738.898,386.89 759.596,399.104 780.294,411.512 800.993,424.915 821.691,436.59 842.39,448.803 \n",
       "  863.088,460.881 883.787,472.826 904.485,485.345 925.184,496.215 945.882,507.453 966.581,518.543 987.279,528.732 1007.98,540.547 1028.68,550.61 1049.37,560.769 \n",
       "  1070.07,571.104 1090.77,581.261 1111.47,591.159 1132.17,601.643 1152.87,611.678 1173.57,622.149 1194.26,630.541 1214.96,639.94 1235.66,649.534 1256.36,659.466 \n",
       "  1277.06,667.997 1297.76,678.18 1318.45,686.794 1339.15,695.912 1359.85,704.34 1380.55,712.129 1401.25,719.676 1421.95,727.639 1442.64,734.326 1463.34,742.013 \n",
       "  1484.04,749.483 1504.74,758.227 1525.44,764.086 1546.14,771.743 1566.84,778.831 1587.53,786.569 1608.23,793.601 1628.93,801.45 1649.63,809.516 1670.33,817.305 \n",
       "  1691.03,825.175 1711.72,831.166 1732.42,837.455 1753.12,844.794 1773.82,850.527 1794.52,857.123 1815.22,863.637 1835.92,871.651 1856.61,877.293 1877.31,884.225 \n",
       "  1898.01,890.437 1918.71,896.731 1939.41,903.189 1960.11,909.001 1980.8,914.668 2001.5,921.838 2022.2,927.209 2042.9,933.179 2063.6,938.989 2084.3,944.351 \n",
       "  2105,950.249 2125.69,956.124 2146.39,962.108 2167.09,967.104 2187.79,972.357 2208.49,977.826 2229.19,983.401 2249.88,988.957 2270.58,994.244 2291.28,999.44 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#ac8d18; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  242.135,100.914 262.833,128.927 283.532,151.842 304.23,178.805 324.929,201.349 345.627,219.404 366.325,244.069 387.024,266.51 407.722,287.363 428.421,308.669 \n",
       "  449.119,331.706 469.818,350.992 490.516,372.842 511.215,391.615 531.913,413.288 552.612,435.41 573.31,454.667 594.008,473.318 614.707,491.877 635.405,508.114 \n",
       "  656.104,531.019 676.802,549.243 697.501,567.969 718.199,587.045 738.898,604.629 759.596,622.924 780.294,640.567 800.993,657.87 821.691,675.622 842.39,695.248 \n",
       "  863.088,712.411 883.787,728.36 904.485,745.062 925.184,760.466 945.882,772.357 966.581,786.928 987.279,803.685 1007.98,815.699 1028.68,828.246 1049.37,838.352 \n",
       "  1070.07,851.287 1090.77,863.262 1111.47,875.636 1132.17,886.694 1152.87,896.945 1173.57,907.73 1194.26,918.541 1214.96,927.845 1235.66,937.323 1256.36,946.724 \n",
       "  1277.06,956.723 1297.76,965.118 1318.45,975.706 1339.15,984.413 1359.85,993.232 1380.55,1002.79 1401.25,1011.16 1421.95,1018.55 1442.64,1027.36 1463.34,1034.28 \n",
       "  1484.04,1040.24 1504.74,1046.86 1525.44,1053.37 1546.14,1058.61 1566.84,1064.28 1587.53,1069.28 1608.23,1074.01 1628.93,1079.87 1649.63,1085.09 1670.33,1090.8 \n",
       "  1691.03,1096.15 1711.72,1101.03 1732.42,1105.14 1753.12,1110.51 1773.82,1116.13 1794.52,1120.34 1815.22,1124.19 1835.92,1129.7 1856.61,1134.34 1877.31,1137.65 \n",
       "  1898.01,1141.49 1918.71,1145.52 1939.41,1150.89 1960.11,1154.89 1980.8,1158.88 2001.5,1162.77 2022.2,1166.95 2042.9,1170.83 2063.6,1175.11 2084.3,1179.64 \n",
       "  2105,1182.59 2125.69,1186.86 2146.39,1190.84 2167.09,1193.98 2187.79,1197.78 2208.49,1201.18 2229.19,1204.75 2249.88,1207.96 2270.58,1211.3 2291.28,1214.77 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#00a9ad; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  242.135,110.544 262.833,148.031 283.532,173.664 304.23,198.311 324.929,227.882 345.627,250.632 366.325,278.498 387.024,308.166 407.722,334.428 428.421,355.786 \n",
       "  449.119,381.802 469.818,402.414 490.516,430.842 511.215,454.083 531.913,475.419 552.612,496.131 573.31,515.391 594.008,538.87 614.707,561.843 635.405,581.912 \n",
       "  656.104,598.321 676.802,619.234 697.501,636.867 718.199,657.902 738.898,677.974 759.596,697.796 780.294,720.375 800.993,737.535 821.691,754.209 842.39,772.763 \n",
       "  863.088,787.613 883.787,804.028 904.485,821.837 925.184,838.978 945.882,852.879 966.581,867.799 987.279,886.035 1007.98,901.177 1028.68,915.662 1049.37,926.564 \n",
       "  1070.07,939.432 1090.77,953.564 1111.47,967.847 1132.17,977.105 1152.87,987.379 1173.57,997.523 1194.26,1007.08 1214.96,1017.61 1235.66,1027.99 1256.36,1036.3 \n",
       "  1277.06,1043 1297.76,1050.12 1318.45,1057.3 1339.15,1067.15 1359.85,1072.49 1380.55,1078.63 1401.25,1084.42 1421.95,1090.58 1442.64,1095.2 1463.34,1099.78 \n",
       "  1484.04,1103.94 1504.74,1108.97 1525.44,1113.88 1546.14,1119.66 1566.84,1124.07 1587.53,1127.62 1608.23,1134.54 1628.93,1140.91 1649.63,1145.17 1670.33,1149.04 \n",
       "  1691.03,1152.83 1711.72,1155.84 1732.42,1158.84 1753.12,1163.05 1773.82,1165.41 1794.52,1168.89 1815.22,1173.94 1835.92,1178.48 1856.61,1182.97 1877.31,1186.42 \n",
       "  1898.01,1189.8 1918.71,1193.4 1939.41,1196.61 1960.11,1200.82 1980.8,1205.14 2001.5,1209.15 2022.2,1213.21 2042.9,1218.46 2063.6,1222.11 2084.3,1227.1 \n",
       "  2105,1230.8 2125.69,1234.74 2146.39,1239.8 2167.09,1243.38 2187.79,1245.69 2208.49,1250.01 2229.19,1253.54 2249.88,1256.66 2270.58,1259.85 2291.28,1262.81 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2902)\" style=\"stroke:#ed5d92; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  242.135,113.088 262.833,145.666 283.532,185.687 304.23,221.794 324.929,256.039 345.627,294.542 366.325,333.341 387.024,359.707 407.722,394.753 428.421,428.934 \n",
       "  449.119,461.988 469.818,492.021 490.516,525.418 511.215,555.044 531.913,580.177 552.612,605.755 573.31,636.504 594.008,661.469 614.707,689.203 635.405,714.28 \n",
       "  656.104,736.615 676.802,759.319 697.501,779.067 718.199,797.979 738.898,818.33 759.596,833.311 780.294,850.233 800.993,864.713 821.691,884.22 842.39,898.779 \n",
       "  863.088,914.841 883.787,928.789 904.485,941.272 925.184,952.954 945.882,961.812 966.581,970.461 987.279,976.81 1007.98,985.308 1028.68,994.148 1049.37,1003.86 \n",
       "  1070.07,1013.11 1090.77,1020.71 1111.47,1027.76 1132.17,1034.54 1152.87,1042.49 1173.57,1051.22 1194.26,1057.79 1214.96,1064.81 1235.66,1072.07 1256.36,1077.53 \n",
       "  1277.06,1085.2 1297.76,1089.46 1318.45,1094.09 1339.15,1097.82 1359.85,1104.43 1380.55,1109.19 1401.25,1116.19 1421.95,1121.72 1442.64,1125.74 1463.34,1130.37 \n",
       "  1484.04,1134.45 1504.74,1139.49 1525.44,1142.36 1546.14,1148.01 1566.84,1150.93 1587.53,1154.87 1608.23,1158.71 1628.93,1161.09 1649.63,1164.96 1670.33,1169.11 \n",
       "  1691.03,1172.89 1711.72,1176.73 1732.42,1184.1 1753.12,1188.01 1773.82,1192.64 1794.52,1197.86 1815.22,1203.37 1835.92,1208.26 1856.61,1212.06 1877.31,1215.77 \n",
       "  1898.01,1219.03 1918.71,1220.95 1939.41,1222.69 1960.11,1226.57 1980.8,1227.32 2001.5,1230.1 2022.2,1232.65 2042.9,1234.45 2063.6,1236.09 2084.3,1239.1 \n",
       "  2105,1242.18 2125.69,1243.39 2146.39,1245.25 2167.09,1248.63 2187.79,1251.11 2208.49,1254.3 2229.19,1258.55 2249.88,1260.28 2270.58,1264.44 2291.28,1266.57 \n",
       "  \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip2900)\" d=\"\n",
       "M1713.01 614.604 L2280.76 614.604 L2280.76 130.764 L1713.01 130.764  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1713.01,614.604 2280.76,614.604 2280.76,130.764 1713.01,130.764 1713.01,614.604 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  1737.01,191.244 1881.01,191.244 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1905.01, 208.744)\" x=\"1905.01\" y=\"208.744\">TD alpha=0.05</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  1737.01,251.724 1881.01,251.724 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1905.01, 269.224)\" x=\"1905.01\" y=\"269.224\">TD alpha=0.1</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#3da44d; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  1737.01,312.204 1881.01,312.204 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1905.01, 329.704)\" x=\"1905.01\" y=\"329.704\">TD alpha=0.15</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#c271d2; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1737.01,372.684 1881.01,372.684 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1905.01, 390.184)\" x=\"1905.01\" y=\"390.184\">MC alpha=0.01</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#ac8d18; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1737.01,433.164 1881.01,433.164 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1905.01, 450.664)\" x=\"1905.01\" y=\"450.664\">MC alpha=0.02</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#00a9ad; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1737.01,493.644 1881.01,493.644 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1905.01, 511.144)\" x=\"1905.01\" y=\"511.144\">MC alpha=0.03</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip2900)\" style=\"stroke:#ed5d92; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1737.01,554.124 1881.01,554.124 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip2900)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1905.01, 571.624)\" x=\"1905.01\" y=\"571.624\">MC alpha=0.04</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = plot()\n",
    "\n",
    "for α in [0.05, 0.1, 0.15]\n",
    "    rms = []\n",
    "    for _ in 1:100\n",
    "        agent = create_TD_agent(α)\n",
    "        hook = RecordRMS()\n",
    "        run(agent, env, StopAfterEpisode(100),hook)\n",
    "        push!(rms, hook.rms)\n",
    "    end\n",
    "    plot!(p, mean(rms), label =\"TD alpha=$α\", linestyle=:dashdot)\n",
    "end\n",
    "\n",
    "for α in [0.01, 0.02, 0.03, 0.04]\n",
    "    rms = []\n",
    "    for _ in 1:100\n",
    "        agent = create_MC_agent(α)\n",
    "        hook = RecordRMS()\n",
    "        run(agent, env, StopAfterEpisode(100),hook)\n",
    "        push!(rms, hook.rms)\n",
    "    end\n",
    "    plot!(p, mean(rms), label =\"MC alpha=$α\")\n",
    "end\n",
    "\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip3300\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip3300)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip3301\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip3300)\" d=\"\n",
       "M180.66 1487.47 L2352.76 1487.47 L2352.76 47.2441 L180.66 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip3302\">\n",
       "    <rect x=\"180\" y=\"47\" width=\"2173\" height=\"1441\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  221.436,1487.47 221.436,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  738.898,1487.47 738.898,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1256.36,1487.47 1256.36,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1773.82,1487.47 1773.82,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2291.28,1487.47 2291.28,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,1404.67 2352.76,1404.67 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,1044.92 2352.76,1044.92 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,685.168 2352.76,685.168 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  180.66,325.419 2352.76,325.419 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,1487.47 2352.76,1487.47 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,1487.47 180.66,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  221.436,1487.47 221.436,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  738.898,1487.47 738.898,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1256.36,1487.47 1256.36,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1773.82,1487.47 1773.82,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2291.28,1487.47 2291.28,1470.19 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,1404.67 206.725,1404.67 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,1044.92 206.725,1044.92 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,685.168 206.725,685.168 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  180.66,325.419 206.725,325.419 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 221.436, 1541.47)\" x=\"221.436\" y=\"1541.47\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 738.898, 1541.47)\" x=\"738.898\" y=\"1541.47\">25</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1256.36, 1541.47)\" x=\"1256.36\" y=\"1541.47\">50</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1773.82, 1541.47)\" x=\"1773.82\" y=\"1541.47\">75</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2291.28, 1541.47)\" x=\"2291.28\" y=\"1541.47\">100</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 1422.17)\" x=\"156.66\" y=\"1422.17\">0.05</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 1062.42)\" x=\"156.66\" y=\"1062.42\">0.10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 702.668)\" x=\"156.66\" y=\"702.668\">0.15</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:end;\" transform=\"rotate(0, 156.66, 342.919)\" x=\"156.66\" y=\"342.919\">0.20</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  242.135,165.431 262.833,256.488 283.532,344.607 304.23,427.425 324.929,507.667 345.627,578.829 366.325,652.862 387.024,717 407.722,781.39 428.421,839.138 \n",
       "  449.119,898.021 469.818,951.362 490.516,1001.82 511.215,1049.65 531.913,1094.56 552.612,1131.71 573.31,1159.07 594.008,1196.41 614.707,1227.7 635.405,1246.29 \n",
       "  656.104,1283.16 676.802,1310.16 697.501,1335.41 718.199,1362.39 738.898,1381.21 759.596,1393.98 780.294,1392.86 800.993,1408.29 821.691,1419.35 842.39,1423.29 \n",
       "  863.088,1423.99 883.787,1442.48 904.485,1446.71 925.184,1438.34 945.882,1437.51 966.581,1441.11 987.279,1435.22 1007.98,1426.7 1028.68,1418.17 1049.37,1415.63 \n",
       "  1070.07,1413.99 1090.77,1408.77 1111.47,1403.36 1132.17,1399.71 1152.87,1412.9 1173.57,1411.02 1194.26,1420.04 1214.96,1421.83 1235.66,1424.48 1256.36,1425.19 \n",
       "  1277.06,1422.07 1297.76,1415.85 1318.45,1424.37 1339.15,1426.32 1359.85,1428.94 1380.55,1424.12 1401.25,1414.91 1421.95,1426.19 1442.64,1424.22 1463.34,1419.83 \n",
       "  1484.04,1411.32 1504.74,1408.67 1525.44,1405.85 1546.14,1405.14 1566.84,1391.98 1587.53,1397.67 1608.23,1387.04 1628.93,1394.72 1649.63,1398.61 1670.33,1387.06 \n",
       "  1691.03,1385.96 1711.72,1386.75 1732.42,1385.29 1753.12,1387.58 1773.82,1382.11 1794.52,1379.61 1815.22,1374.79 1835.92,1387.33 1856.61,1388.21 1877.31,1389.83 \n",
       "  1898.01,1383.53 1918.71,1388.46 1939.41,1401.95 1960.11,1398.06 1980.8,1393.79 2001.5,1390.64 2022.2,1384.96 2042.9,1381.1 2063.6,1380.19 2084.3,1374.97 \n",
       "  2105,1380.76 2125.69,1379.62 2146.39,1381.19 2167.09,1380.11 2187.79,1386.47 2208.49,1392.78 2229.19,1390.13 2249.88,1375.3 2270.58,1378.18 2291.28,1375.71 \n",
       "  \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3302)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  242.135,88.0053 262.833,139.415 283.532,204.277 304.23,232.237 324.929,300.287 345.627,351.058 366.325,403.074 387.024,467.129 407.722,536.339 428.421,601.552 \n",
       "  449.119,647.717 469.818,696.179 490.516,741.742 511.215,778.191 531.913,807.831 552.612,850.115 573.31,865.049 594.008,895.042 614.707,905.906 635.405,922.562 \n",
       "  656.104,938.767 676.802,961.567 697.501,978.239 718.199,993.077 738.898,993.619 759.596,1006.61 780.294,1008.08 800.993,1016.66 821.691,1023.23 842.39,1032.16 \n",
       "  863.088,1039.35 883.787,1046.92 904.485,1055.27 925.184,1062.96 945.882,1072.27 966.581,1078.77 987.279,1082.96 1007.98,1088.31 1028.68,1095.45 1049.37,1106.28 \n",
       "  1070.07,1120.2 1090.77,1129.28 1111.47,1140.98 1132.17,1142.97 1152.87,1149.29 1173.57,1157.81 1194.26,1163.85 1214.96,1173.82 1235.66,1177.79 1256.36,1186.36 \n",
       "  1277.06,1192.25 1297.76,1194.7 1318.45,1199.01 1339.15,1202.11 1359.85,1204.98 1380.55,1208.4 1401.25,1210.65 1421.95,1216.27 1442.64,1217.14 1463.34,1217.51 \n",
       "  1484.04,1222.05 1504.74,1227.45 1525.44,1231.24 1546.14,1233.81 1566.84,1240.21 1587.53,1242.79 1608.23,1244.11 1628.93,1248.46 1649.63,1250.47 1670.33,1252.45 \n",
       "  1691.03,1251.11 1711.72,1251.91 1732.42,1254.73 1753.12,1256.13 1773.82,1260.72 1794.52,1264.15 1815.22,1268.57 1835.92,1274.27 1856.61,1280.43 1877.31,1287.22 \n",
       "  1898.01,1291.45 1918.71,1295.73 1939.41,1300.35 1960.11,1303.07 1980.8,1307.99 2001.5,1312.84 2022.2,1317.68 2042.9,1321.49 2063.6,1325.21 2084.3,1327.89 \n",
       "  2105,1332.16 2125.69,1334.42 2146.39,1338.84 2167.09,1343.28 2187.79,1350.79 2208.49,1354.77 2229.19,1356.4 2249.88,1359.29 2270.58,1360.59 2291.28,1362.19 \n",
       "  \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip3300)\" d=\"\n",
       "M1739.78 312.204 L2280.76 312.204 L2280.76 130.764 L1739.78 130.764  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1739.78,312.204 2280.76,312.204 2280.76,130.764 1739.78,130.764 1739.78,312.204 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" stroke-dasharray=\"16, 8, 2, 8\" points=\"\n",
       "  1763.78,191.244 1907.78,191.244 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1931.78, 208.744)\" x=\"1931.78\" y=\"208.744\">TD alpha=0.1</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip3300)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1763.78,251.724 1907.78,251.724 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip3300)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 1931.78, 269.224)\" x=\"1931.78\" y=\"269.224\">MC alpha=0.1</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = plot()\n",
    "\n",
    "rms = []\n",
    "for _ in 1:100\n",
    "    agent = create_TD_agent(0.1)\n",
    "    hook = RecordRMS()\n",
    "    run(agent, env, StopAfterEpisode(100),hook)\n",
    "    push!(rms, hook.rms)\n",
    "end\n",
    "plot!(p, mean(rms), label =\"TD alpha=0.1\", linestyle=:dashdot)\n",
    "\n",
    "\n",
    "rms = []\n",
    "for _ in 1:100\n",
    "    agent = create_MC_agent(0.1)\n",
    "    hook = RecordRMS()\n",
    "    run(agent, env, StopAfterEpisode(100),hook)\n",
    "    push!(rms, hook.rms)\n",
    "end\n",
    "plot!(p, mean(rms), label =\"MC alpha=0.1\")\n",
    "\n",
    "p"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.3.0",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
